package com.infindo.appcreate.zzyj.controller.scheduler;

import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.annotation.Resource;

import org.apache.commons.httpclient.NameValuePair;
import org.json.JSONException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.web.bind.annotation.RequestMapping;

import com.infindo.appcreate.zzyj.entity.Activity;
import com.infindo.appcreate.zzyj.entity.ActivityTalk;
import com.infindo.appcreate.zzyj.entity.Expert;
import com.infindo.appcreate.zzyj.entity.ExpertProject;
import com.infindo.appcreate.zzyj.entity.Infomation;
import com.infindo.appcreate.zzyj.entity.Project;
import com.infindo.appcreate.zzyj.entity.ProjectComment;
import com.infindo.appcreate.zzyj.entity.ProjectHot;
import com.infindo.appcreate.zzyj.entity.ProjectSupporter;
import com.infindo.appcreate.zzyj.httpclient.HttpProtocolHandler;
import com.infindo.appcreate.zzyj.httpclient.HttpRequest;
import com.infindo.appcreate.zzyj.httpclient.HttpResponse;
import com.infindo.appcreate.zzyj.httpclient.HttpResultType;
import com.infindo.appcreate.zzyj.service.ZzyjSpiderService;
import com.infindo.appcreate.zzyj.util.StringUtil;



@RequestMapping(value = "/scheduler")
public class SchedulerManager {


    @Resource
    private ZzyjSpiderService zzyjSpiderService;

    
    private final String ZZYJ_SITE = "http://www.zhongzhiyunji.com";
    private final String ZZYJ_PROJECT_LIST = "http://www.zhongzhiyunji.com/project/display/index/time/desc/";
    private final String ZZYJ_PROJECT_SUPPORTER = "http://www.zhongzhiyunji.com/project/display_action/ajax_top/";
    private final String ZZYJ_PROJEDT_COMMENT = "http://www.zhongzhiyunji.com/project/display_action/ajax_reply/";
    private final String ZZYJ_PROJECT_HOT = "http://www.zhongzhiyunji.com/project/display/index/hot/desc/";
    private final String ZZYJ_EXPERT_LIST = "http://www.zhongzhiyunji.com/expert/index/";
    private final String ZZYJ_EXPERT_DETAIL = "http://www.zhongzhiyunji.com/user/qzone/";
    private final String ZZYJ_EXPERT_DESC = "http://www.zhongzhiyunji.com/ajax/getInfo";
    private final String ZZYJ_EXPERT_PROJECT_SUPPORT = "http://www.zhongzhiyunji.com/user/qzone/";
    private final String ZZYJ_SERVICE = "http://www.zhongzhiyunji.com/service/fund/index/1";
    private final String ZZYJ_SERVICE_INFORM = "http://www.zhongzhiyunji.com/news/info/43";
    private final String ZZYJ_SERVICE_DECLVIDEOS = "http://www.zhongzhiyunji.com/news/info/44";
    private final String ZZYJ_ACTIVITY_LIST = "http://www.zhongzhiyunji.com/activity/index/index/";
    private final String ZZYJ_ACTIVITY_DETAIL = "http://www.zhongzhiyunji.com/activity/index/info/";
    private final String ZZYJ_INFOMATION_LIST = "http://www.zhongzhiyunji.com/news/lists/createdAt/";
    private final String ZZYJ_INFOMATION_DETAIL = "http://www.zhongzhiyunji.com/news/info/";
    
    
    private final Integer CON_TIMEOUT = 60000;
    private final Integer EXPERT_PERPAGE_COUNT = 5;
    private final Integer PAGE_COUNT = 5;
    
    List voList = new ArrayList();
    private void spideData() {
        
         voList = new ArrayList(); 
         System.out.println(">>>>>>>>>> spide project begin..................");
         spideProject(voList);
         spideProjectHot(voList);
         System.out.println(">>>>>>>>>> spide project end....................");
            
         //spider expert info
         System.out.println(">>>>>>>>>> spide expert begin..................");
         spideExpertInfo(voList);
         System.out.println(">>>>>>>>>> spide expert end....................");
            
         //spider Service info
         System.out.println(">>>>>>>>>> spide Service begin..................");
         spideService(voList);
         System.out.println(">>>>>>>>>> spide Service end....................");
            
         //--spider activity list
         System.out.println(">>>>>>>>>> spide activity begin..................");
         spideActivity(voList);
         System.out.println(">>>>>>>>>> spide activity end....................");
             
         //spide infomation code list
         System.out.println(">>>>>>>>>> spide infomation begin..................");
         spideInfomation(voList);
         System.out.println(">>>>>>>>>> spide infomation end....................");
         
    }
    
    /**
     * spide infomation
     * @param voList
     * @return
     */
    private int spideInfomation(List voList){
        
        List<Infomation> infoList = new ArrayList<Infomation>();
        
        Map<String,String> infoCodeMap = new HashMap<String,String>();
        List<String> infoCodeList = new ArrayList<String>();
        List<String> infoImageList = new ArrayList<String>();
        List<String> infoTimeList = new ArrayList<String>();
        List<String> infoOriginList = new ArrayList<String>();
        
        Elements infoEls = null;
        Document doc = null;
        
        boolean shouldBreak = false;
        int pagingCount = 1;
        while(true){//for pagination
            
            if(shouldBreak){
                break;
            }
            
            try {
                doc = Jsoup.connect(ZZYJ_INFOMATION_LIST+pagingCount).timeout(CON_TIMEOUT).get();
                
                infoEls = doc.select("div.news_box div[class *= clearfix mt10] img");
                if(infoEls.size() > 0){
                    String curInfoCode = "";
                    for(Element el: infoEls){
                        curInfoCode = getCodeFromUrl(el.parent().attr("href"));
                        
                        if(infoCodeMap.containsKey(curInfoCode)){
                            shouldBreak = true;
                            curInfoCode = null;
                            break;
                        }else{
                            infoCodeMap.put(curInfoCode, curInfoCode);
                            infoCodeList.add(curInfoCode);
                            infoImageList.add(el.attr("src"));
                            
                            infoEls = el.parent().parent().select("div.news_con p span");
                            if(infoEls.size() > 0){
                                infoOriginList.add(infoEls.get(1).text());
                                infoTimeList.add(getPostStringOfColon(infoEls.get(2).text()));
                            }
                        }
                    }
                }else{
                    break;
                }
                
                pagingCount ++;
                
            }catch(IOException e){
                e.printStackTrace();
                break;
            }
        }
        
        //loop to get infomation detail
        Long iCode = 0l;
        String iTitle = "";
        String iImage = "";
        String iOrigin = "";
        Date iTime = null;
        String iDesc = "";
        
        int _c = 0;
        for(String code : infoCodeList){
            iCode = Long.valueOf(code);
            iImage = infoImageList.get(_c);
            iOrigin = infoOriginList.get(_c);
            iTime = getTimeFromStrByF2(infoTimeList.get(_c),"yyyy.MM.dd");
            _c++;
            
            try {
                doc = Jsoup.connect(ZZYJ_INFOMATION_DETAIL+code).timeout(CON_TIMEOUT).get();
                
                infoEls = doc.select("div.news_info_left div.title");
                if(infoEls.size() > 0){
                    iTitle = infoEls.first().text();
                }
                
                infoEls = doc.select("div.news_detail");
                if(infoEls.size() > 0){
                    iDesc = infoEls.first().html();
                }
                
                voList.add(new Infomation(iCode, iTitle, iImage, iOrigin, iTime, iDesc));
                //System.out.println(iCode+";"+iTitle+";"+iTitle+";"+iOrigin+";"+iTime+";");
                
            } catch (IOException e) {
                e.printStackTrace();
                return 0;
            }
            
        }
        
        //this.saveSpiderData(infoList);
        
        return 1;
    }
    
    /**
     * spide Activity info
     * @param voList
     * @return
     */
    private int spideActivity(List voList){
        
        List<Activity> activityList = new ArrayList<Activity>();
        List<ActivityTalk> activityTalkList = new ArrayList<ActivityTalk>();
        
        Map<String,String> actCodeMap = new HashMap<String,String>();
        List<String> actCodeList = new ArrayList<String>();
        List<String> actListImageList = new ArrayList<String>();
        
        Document doc = null;
        Elements aEls = null;
        
        int pagingCount = 1;
        boolean shouldBreak = false;
        while(true){//for pagination
            
            if(shouldBreak){
                break;
            }
            
            try {
                doc = Jsoup.connect(ZZYJ_ACTIVITY_LIST+pagingCount).timeout(CON_TIMEOUT).get();                
                aEls = doc.select("div.project_left li div.showpic a[href *= index/info]");
                
                if(aEls.size() > 0){
                    String curActCode = "";
                    for(Element el:aEls){
                        curActCode = getCodeFromUrl(el.attr("href"));
                        
                        if(actCodeMap.containsKey(curActCode)){
                            shouldBreak = true;
                            curActCode = null;
                            break;
                        }else{
                            actCodeMap.put(curActCode, curActCode);
                            actCodeList.add(curActCode);
                            actListImageList.add(el.select("img").first().attr("src"));
                        }
                    }
                }else{
                    break;
                }
                
                pagingCount ++;
                
            }catch(IOException e){
                e.printStackTrace();
            }
        }
        
        //loop to get activity detail
        String aCode = "";
        String aTitle = "";
        Date aTime = null;
        String aAddress = "";
        String aImage = "";
        String aImages = "";
        String aDesc = "";
        
        int _c = 0;
        for(String code :actCodeList){
            aImages = "";
            aCode = code;
            aImage = actListImageList.get(_c++);
            try {
                doc = Jsoup.connect(ZZYJ_ACTIVITY_DETAIL+code).timeout(CON_TIMEOUT).get();
                
                aEls = doc.select("h4.activity_info_title");
                if(aEls.size() > 0){
                    aTitle = aEls.first().text();
                }
                
                aEls = doc.select("ul.activity_info_sub li");
                if(aEls.size() > 1){
                    
                    aAddress = getPostStringOfColon(aEls.get(0).text());
                    if(StringUtil.isNotBlank(aEls.get(1).text())){
                        aTime = getTimeFromStrByF2(getPostStringOfColon(aEls.get(1).text()),"yyyy.MM.dd");
                    }
                }
                
                aEls = doc.select("h4.activity_info_title");
                if(aEls.size() > 0){
                    aTitle = aEls.first().text();
                }
                
                aEls = doc.select("div.act_box1");
                if(aEls.size() > 0){
                    aDesc = aEls.first().html();
                }
                
                aEls = doc.select("div.act_phone li img");
                for(Element el :aEls){
                    aImages += el.attr("src")+",";
                }
                if(aImages.endsWith(",")){
                    aImages = aImages.substring(0,aImages.length() - 1);
                }
                
                voList.add(new Activity(aCode, aTitle, aTime, aAddress, aImage,aImages, aDesc));
                //System.out.println(aCode+";"+aTitle+";"+aTime+";"+aAddress+";"+aImage+";"+aImages+";"+aDesc);
                
                //--spide activity talk
                spideActTalk(voList,code);
                
                
            } catch (IOException e) {
                e.printStackTrace();
                return 0;
            }
            
        }
        
        //this.saveSpiderData(activityList);
        //this.saveSpiderData(activityTalkList);
        
        return 1;
    }
    
    private int spideActTalk(List voList, String actCode){
        
        String atActCode = actCode;
        String atExpertCode = "";//public or expert code
        String atImage = "";
        String atNickName = "";
        Date atTime = null;
        String atDesc = "";
        
        Document doc;
        Elements aEls;
        
        try {
            doc = Jsoup.connect(ZZYJ_ACTIVITY_DETAIL+actCode+"#discus").timeout(CON_TIMEOUT).get();
            aEls = doc.select("div.activity_discus");
            
            for(Element el :aEls){
                
                aEls = el.select("img.avater");
                if(aEls.size() > 0){
                    atImage = aEls.first().attr("src");
                }
                
                aEls = el.select("p a[href *= /user/qzone]");
                if(aEls.size() > 0){
                    atExpertCode = getCodeFromUrl(aEls.first().attr("href"));
                    atNickName = aEls.first().text();
                }
                
                aEls = el.select("div.comment_arae p");
                if(aEls.size() > 0){
                    
                    if(StringUtil.isNotBlank(aEls.get(1).text())){
                        atTime = getTimeFromStrByF2(aEls.get(1).text(),"yyyy.MM.dd hh:mm:ss");
                    }
                    atDesc = aEls.get(2).text();
                }
                
                voList.add(new ActivityTalk(atActCode, atExpertCode, atImage, atNickName, atTime, atDesc));
                //System.out.println(atActCode+";"+atExpertCode+";"+atImage+";"+atNickName+";"+atTime+";"+atDesc);
            }
            
        } catch (IOException e) {
            e.printStackTrace();
            return 0;
        }
        return 1;
    }
    
    /**
     * spide service
     * @param voList
     * @return
     */
    private int spideService(List voList){
        
        List<com.infindo.appcreate.zzyj.entity.Service> serviceList= new ArrayList<com.infindo.appcreate.zzyj.entity.Service>();
        
        String sImage = "";
        String sTitle = "";
        String sOrgan = "";
        String sDesc = "";
        String sDeclInforms = "";//多个 dot 隔开, 声明通知
        String sOpeDetails = "";//多个 dot 隔开，操作细节
        String sDeclVideos = "";//多个 dot 隔开,声明视频
        
        Document doc ;
        Elements els;
        
        try {
            doc = Jsoup.connect(ZZYJ_SERVICE).timeout(CON_TIMEOUT).get();
            els = doc.select("div.service_banner");
            if(els.size() > 0){
               Element el = els.first();
               
               els = el.select("div.fl img");
               if(els.size() > 0){
                   sImage = els.first().attr("src");
               }
               
               els = el.select("div.service_banner_right h4");
               if(els.size() > 0){
                   sTitle = els.first().text();
               }
               
               els = el.select("div[class *= con]");
               if(els.size() > 0){
                   sDesc = els.first().text();
               }
               
               els = el.select("p span");
               if(els.size() > 0){
                   sOrgan = els.get(1).text();
               }
               
               els = doc.select("div.service_main p img");
               if(els.size() > 0){
                   sDeclInforms = els.first().attr("src");
                   sOpeDetails = els.last().attr("src");
               }
               
            }
            
            //--spide service declare videos
            doc = Jsoup.connect(ZZYJ_SERVICE_DECLVIDEOS).timeout(CON_TIMEOUT).get();
            els = doc.select("div.news_detail p");
            if(els.size() > 0){
                sDeclVideos = cutStrByReg(els.first().text(),"http");
            }
            
            voList.add(new com.infindo.appcreate.zzyj.entity.Service(sImage, sTitle, sOrgan, sDesc, sDeclInforms, sOpeDetails, sDeclVideos));
            //System.out.println(sImage+";"+sTitle+";"+sOrgan+";"+sDesc+";"+sDeclInforms+";"+sOpeDetails+";"+sDeclVideos);
            
        } catch (IOException e) {
            e.printStackTrace();
            return 0;
        }
        
        //this.saveSpiderData(serviceList);
        
        return 1;
    }
    
    /**
     * spide expert info
     * @param voList
     * @return
     */
    private int spideExpertInfo(List voList){

        List<String> expertCodeList = new ArrayList<String>();
        
        List<Expert> experList = new ArrayList<Expert>();
        List<ExpertProject> expertSupList = new ArrayList<ExpertProject>();
        
        Document doc;
        Elements els ;
        
        //--spider expert list
        Map<String,String> preExpertCodeMap = new HashMap<String,String>();
        boolean shouldBreak = false;
        int pagingCount = 1;
        while(true){//for pagination
            
            if(shouldBreak){
                break;
            }
            
            try {
                doc = Jsoup.connect(ZZYJ_EXPERT_LIST+pagingCount).timeout(CON_TIMEOUT).get();                
                els = doc.select("div.expert_col img[src *= avatar]");
                
                if(els.size() > 0){
                    String curExpertCode = "";
                    for(Element el:els){
                        curExpertCode = getCodeFromUrl(el.parent().attr("href"));
                        
                        if(preExpertCodeMap.containsKey(curExpertCode)){
                            shouldBreak = true;
                            preExpertCodeMap = null;
                            break;
                        }else{
                            preExpertCodeMap.put(curExpertCode, curExpertCode);
                            expertCodeList.add(curExpertCode);
                        }
                    }
                }else{
                    break;
                }
                
                pagingCount ++;
                
            }catch(IOException e){
                e.printStackTrace();
                break;
            }
        }
        
        //loop to get expert detail
        String _code = "";
        String _name = "";
        String _image = "";
        String _desc = "";
        String _authIcon = "";//专家认证图标 V
        Integer _authIconType = 0;//0 blue v; 1 yellow v;
        String _weiBo = "";
        
        for(String code :expertCodeList){
            _code = code;
            try {
                doc = Jsoup.connect(ZZYJ_EXPERT_DETAIL+code).timeout(CON_TIMEOUT).get();
                
                els = doc.select("img.bigavater");
                if(els.size() > 0){
                    _image = els.first().attr("src");
                }
                
                els = doc.select("span[class *= ucname showverifyinfo]");
                if(els.size() > 0){
                    Element el = els.first();
                    _name = el.text();
                    
                    _authIcon = ZZYJ_SITE+el.select("img").first().attr("src");
                }
                
                els = doc.select("div[class *= fr ucright] p");
                if(els.size() > 0){
                    Element el = els.last();
                    _weiBo = cutStrByReg(el.text(),"http");
                }
                
                HttpProtocolHandler httpProtocolHandler = HttpProtocolHandler.getInstance();
                HttpRequest req = new HttpRequest(HttpResultType.STRING);
                req.setCharset("UTF-8");
                NameValuePair[] nameValuePair = new NameValuePair[1];
                nameValuePair[0] = new NameValuePair("userId",code);
                req.setParameters(nameValuePair);
                req.setMethod(HttpRequest.METHOD_POST);
                req.setUrl(ZZYJ_EXPERT_DESC);
                HttpResponse response = httpProtocolHandler.execute(req,"","");
                _desc = response.getStringResult();
                if(StringUtil.isNotBlank(_desc)){
                    try {
                        org.json.JSONObject infoJason = new org.json.JSONObject(_desc);
                        _desc = (String) infoJason.get("content");
                        
                    } catch (JSONException e) {
                        e.printStackTrace();
                    } 
                }
                
                voList.add(new Expert(_code, _name, _image, _desc, _authIcon, _weiBo));
                //System.out.println(_weiBo+";"+_desc+";"+_authIcon+";"+_code+";"+_image+";"+_name);
                
                
               //spider expert suport project
                spideExpertSupport(voList,code);
                
                //spider expert arise project
                spideExpertArise(voList,code);
                
            } catch (IOException e) {
                e.printStackTrace();
                return 0;
            }
        }
        
        //this.saveSpiderData(experList);
        //this.saveSpiderData(expertSupList);
        
        return 1;
    }
    
    private int spideExpertArise(List voList, String expertCode){
        
        Document doc;
        Elements els;
        
        try {
            doc = Jsoup.connect(ZZYJ_EXPERT_PROJECT_SUPPORT+expertCode+"/arise").timeout(CON_TIMEOUT).get();
            els = doc.select("div.project_left li div.showpic a");
            if(els.size() > 0){
                
                String _expertCode = expertCode;
                String _projectCode;
                Integer _type = 1;//0 支持的; 1 发起的
                
                for(Element el :els){
                    _projectCode = getCodeFromUrl(el.attr("href"));
                    voList.add(new ExpertProject(_expertCode, _projectCode, _type));
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
            return 0;
        }
        return 1;
    }
    
    private int spideExpertSupport(List voList, String expertCode){
        
        Document doc;
        Elements els;
        
        try {
            doc = Jsoup.connect(ZZYJ_EXPERT_PROJECT_SUPPORT+expertCode+"/support").timeout(CON_TIMEOUT).get();
            els = doc.select("div.project_left li div.showpic a");
            if(els.size() > 0){
                
                String _expertCode = expertCode;
                String _projectCode;
                Integer _type = 0;//0 支持的; 1 发起的
                
                for(Element el :els){
                    _projectCode = getCodeFromUrl(el.attr("href"));
                    voList.add(new ExpertProject(_expertCode, _projectCode, _type));
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
            return 0;
        }
        return 1;
    }
   
    
    private int spideProjectHot(List voList){
        Document doc = null;
        Document docSub = null;
        
        Map<String,String> proCodeMap = new HashMap<String,String>();
        //spider project list
        int pagingCount = 1;
        boolean shouldBreak = false;
        while(true){//for pagination
            /*if(pagingCount > PAGE_COUNT){
                break;
            }*/
            if(shouldBreak){
                break;
            }
            
            try {
                doc = Jsoup.connect(ZZYJ_PROJECT_HOT+pagingCount).timeout(CON_TIMEOUT).get();
                
                Elements els = doc.select("div.project_left ul li");
                
                if(els.size() > 0){
                    String projectCode = "";
                    for(Element el:els){
                        
                        Elements divPicAEls = el.select("div.showpic a");
                        if(divPicAEls.size() > 0){
                            Element picAEl = divPicAEls.first();
                            projectCode = getCodeFromUrl(picAEl.attr("href"));
                            
                            if(proCodeMap.containsKey(projectCode)){
                                shouldBreak = true;
                                proCodeMap = new HashMap<String,String>();
                                break;
                            }else{
                                proCodeMap.put(projectCode, projectCode);
                            }
                        }
                  
                        voList.add(new ProjectHot(projectCode));
                        //System.out.println(projectCode);
                    }
                    
                }else{
                    break;
                }
                
                pagingCount ++;
            } catch (IOException e) {
                e.printStackTrace();
                return 0;
            } 
        }
        return 1;
    }
    
    /**
     * spide project info
     * @param voList
     * @return
     */
    private int spideProject(List voList){
        Document doc = null;
        Document docSub = null;
        
        Map<String,String> proCodeMap = new HashMap<String,String>();
        List<Project> proList = new ArrayList<Project>();
        List<ProjectSupporter> proSupList = new ArrayList<ProjectSupporter>();
        List<ProjectComment> proCommList = new ArrayList<ProjectComment>();
        //spider project list
        int pagingCount = 1;
        boolean shouldBreak = false;
        while(true){//for pagination
            /*if(pagingCount > PAGE_COUNT){
                break;
            }*/
            if(shouldBreak){
                break;
            }
            
            try {
                doc = Jsoup.connect(ZZYJ_PROJECT_LIST+pagingCount).timeout(CON_TIMEOUT).get();
                
                Elements els = doc.select("div.project_left ul li");
                
                if(els.size() > 0){
                    for(Element el:els){
                        
                        String listImage = "";
                        String code = "";
                        String title = "";
                        Date time = null;
                        String place = "";
                        Integer score = 0;
                        Integer expertScore = 0;
                        Integer crowdScore = 0;
                        String proDetailRelUrl = "";
                        
                        String user = "";
                        String promoteImages = "";
                        Integer promoteType = 0;
                        String desc = "";
                        
                        Elements divPicAEls = el.select("div.showpic a");
                        if(divPicAEls.size() > 0){
                            Element picAEl = divPicAEls.first();
                            proDetailRelUrl = picAEl.attr("href");
                            code = getCodeFromUrl(proDetailRelUrl);
                            
                            if(proCodeMap.containsKey(code)){
                                shouldBreak = true;
                                proCodeMap = new HashMap<String,String>();
                                break;
                            }else{
                                proCodeMap.put(code, code);
                            }
                            
                        }
                        
                        Elements aH4TitleEls = el.select("a h4.box_title");
                        if(aH4TitleEls.size() > 0){
                            Element aH4TitleEl = aH4TitleEls.first();
                            title = aH4TitleEl.ownText();
                        }
                        
                        Elements divMt10Els = el.select("div.mt10");
                        if(divMt10Els.size() > 0){
                            Element divMt10El = divMt10Els.first();
                            String timeStr = divMt10El.ownText();
                            time = getTimeFromStrByF(timeStr,"yyyy-MM-dd");
                        }
                        
                        Elements spanM15AEls = el.select("span a.c96");
                        if(spanM15AEls.size() > 0){
                            Element spanM15AEl = spanM15AEls.first();
                            place = spanM15AEl.ownText();
                        }
                        
                        Elements scoreEls = el.select("span[class *= ml15 fs24 c0078B6 italic fwb]");
                        if(scoreEls.size() > 0){
                            Element scoreEl = scoreEls.first();
                            score = Integer.valueOf(scoreEl.ownText().trim());
                        }
                        
                        Elements spanExpertEls = el.select("span[class ~= expert]").parents().first().select("span");
                        if(spanExpertEls.size() > 0){
                            Element spanExpertEl = spanExpertEls.last();
                            expertScore = Integer.valueOf(spanExpertEl.ownText().trim());
                        }
                        
                        Elements spanPublicEls = el.select("span[class ~= public]").parents().first().select("span");
                        if(spanPublicEls.size() > 0){
                            Element spanPublicEl = spanPublicEls.last();
                            crowdScore = Integer.valueOf(spanPublicEl.ownText().trim());
                        }
                        
                        Elements divPicImgEls = el.select("div.showpic img");
                        if(divPicImgEls.size() > 0){
                            Element divPicImgEl = divPicImgEls.first();
                            listImage = divPicImgEl.attr("src");
                        }
                       

                        docSub = Jsoup.connect(ZZYJ_SITE+proDetailRelUrl).timeout(CON_TIMEOUT).get();
                        Elements divAUserEls = docSub.select("a[href *= user/qzone]");
                        if(divAUserEls.size() > 0){
                            Element divAUserEl = divAUserEls.first();
                            user = divAUserEl.ownText();
                        }
                        
                        Elements promoteImagesEls = docSub.select("ul.ad-thumb-list li");
                        if(promoteImagesEls.size() > 0){
                            String promoteSrc;
                            promoteType = 0;
                            if(promoteImagesEls.size() > 0){
                                for(Element el1: promoteImagesEls){
                                    promoteSrc = el1.select("a").first().attr("href");
                                    promoteImages += promoteSrc+",";
                                }
                                if(promoteImages.endsWith(",")){
                                    promoteImages.substring(0, promoteImages.length()-1);
                                }
                            } 
                        }
                        promoteImagesEls = docSub.select("div.projectflv_banner_left embed");
                        if(promoteImagesEls.size() > 0){
                            promoteType = 1;
                            promoteImages = promoteImagesEls.first().attr("src");
                        }
                        
                        
                        Elements descEls = docSub.select("div.project_info");
                        if(descEls.size() > 0){
                            Element descEl = descEls.first();
                            desc = descEl.html();
                        }
                        
                        voList.add(new Project(code, title, time, user, place, score, expertScore,crowdScore, listImage, promoteImages, promoteType, desc));
                        //System.out.println(score);
                        
                        //spide project supporter
                        spideProSupporter(voList, code);
                         
                         //spider project comment
                        spideProComment(voList, code);
                    }
                    
                }else{
                    break;
                }
                
                pagingCount ++;
            } catch (IOException e) {
                e.printStackTrace();
                return 0;
            } 
            
        }
        
        //this.saveSpiderData(proList);
        //this.saveSpiderData(proSupList);
        //this.saveSpiderData(proCommList);
        
        return 1;
    }
    
    private int spideProComment(List voList, String proCode){
        
        String cProjectCode = proCode;
        String cExpertCode = "";
        Date cTime = null;      
        String cDesc = "";
        Integer cType = 0;//0 expert; 1 public 
        String cImage = "";
        String cAuthIcon = "";
        String cNickName = "";
        
        Document doc = null;
        Elements els = null;
        
        try {
            int cc = 1;
            while(true){
                
                 doc = Jsoup.connect(ZZYJ_PROJEDT_COMMENT+proCode+"/public/"+cc).timeout(CON_TIMEOUT).get();
               
                 els = doc.select("div.commbox");
                 cType = 1;
                 if(els.select("div.fl").size() > 0){
                     
                     for(Element _el :els){
                         cImage = _el.child(0).select("img").first().attr("src");
                         if(!cImage.contains("http")){
                             cImage = ZZYJ_SITE+cImage;
                         }
                         els = _el.child(1).select("p");
                         if(els.size() > 0){
                             
                             Elements __els = els.first().select("a");
                             if(__els.size() > 0){
                                 cExpertCode = getCodeFromUrl(__els.first().attr("href"));
                                 cNickName = __els.first().text();
                             }
                             cTime = getTimeFromStrByF2(els.get(1).text(),"yyyy-MM-dd hh:mm:ss");
                             cDesc = els.last().text();
                         }
                         
                         voList.add(new ProjectComment(cProjectCode, cExpertCode, cTime, cDesc, cType,cImage.trim(), cAuthIcon.trim(), cNickName.trim()));
                         //System.out.println(cImage+";"+cAuthIcon+";"+cNickName+";"+cProjectCode+";"+cExpertCode+";"+cTime+";"+cDesc+";"+cType);
                     }
                     
                 }else{
                     break;
                 }
                 
                 cc++;
             }
             
             cc = 1;
             while(true){
                 
                 if(cc > PAGE_COUNT){
                     break;
                 }
                 
                 doc = Jsoup.connect(ZZYJ_PROJEDT_COMMENT+proCode+"/expert/"+cc).timeout(CON_TIMEOUT).get();
                 els = doc.select("div.commbox");
                 cType = 0;
                 if(els.select("div.fl").size() > 0){
                     
                     for(Element _el :els){
                         cImage = _el.child(0).select("img").first().attr("src");
                         if(!cImage.contains("http")){
                             cImage = ZZYJ_SITE+cImage;
                         }
                         els = _el.child(1).select("p");
                         if(els.size() > 0){
                             
                             Elements __els = els.first().select("a");
                             if(__els.size() > 0){
                                 cExpertCode = this.getCodeFromUrl(__els.first().attr("href"));
                                 cNickName = __els.first().text();
                                 if(__els.first().select("img").size() > 0){
                                     cAuthIcon = ZZYJ_SITE+__els.first().select("img").first().attr("src");
                                 }
                                 
                             }
                             cTime = getTimeFromStrByF2(els.get(1).text(),"yyyy-MM-dd hh:mm:ss");
                             
                             cDesc = els.last().text();
                         }
                         
                         voList.add(new ProjectComment(cProjectCode, cExpertCode, cTime, cDesc, cType,cImage.trim(), cAuthIcon.trim(), cNickName.trim()));
                         //System.out.println(">>>comment>>  "+cAuthIcon+";"+cImage+";"+cNickName+";"+cProjectCode+";"+cExpertCode+";"+cTime+";"+cDesc+";"+cType);
                     }
                     
                 }else{
                     break;
                 }
                 
                 cc++;
             }
        } catch (IOException e) {
            e.printStackTrace();
            return 0;
        }
        
        return 1;
        
    }
    
    private int spideProSupporter(List voList, String proCode){
        String _projectCode = proCode;
        String _expertCode = "";
        Date _time = null;
        Integer _score = 0;
        Integer _type = 0;//0 expert; 1 public;
        String _image = "";
        String _nickName = "";
        String _authIcon = "";
        
         Document doc = null;
         Elements els = null;
         int c = 1;
         
         try {
         
           while(true){
                 
               if(c > PAGE_COUNT){
                   break;
               }
               
               doc = Jsoup.connect(ZZYJ_PROJECT_SUPPORTER+proCode+"/public/"+c).timeout(CON_TIMEOUT).get();
             
              els = doc.select("div.commbox");
              _type = 1;
              if(els.select("div.fl").size() > 0){
                  
                  for(Element _el :els){
                      
                      els = _el.select("img.avater");
                      if(els.size() > 0){
                          _image = els.first().attr("src");
                          if(_image.contains("http")){
                              _image = ZZYJ_SITE + _image.trim(); 
                          }
                      }
                      
                      els = _el.select("div").last().select("p");
                      if(els.size() > 0){
                          _time = getTimeFromStrByF2(els.first().text(),"yyyy-MM-dd hh:mm:ss");
                      }
                      
                      els = _el.select("a[href *= /user/qzone]");
                      if(els.size() > 0){
                          _nickName = els.first().text();
                          _expertCode = this.getCodeFromUrl(els.first().attr("href"));
                          
                          els = els.first().parent().parent().select("p").last().select("span");
                          if(els.size() > 0){
                              if(StringUtil.isNotBlank(els.first().text().trim())){
                                  _score = Integer.valueOf(els.first().text().trim());  
                              }
                          }
                      }
                      
                      voList.add(new ProjectSupporter(_projectCode, _expertCode, _time, _score, _type,_image.trim(),_authIcon.trim(),_nickName.trim()));
                      //System.out.println(_nickName+";"+_authIcon+";"+_image+";"+_projectCode+";"+_expertCode+";"+_time+";"+_score+";"+_type);
                  }
                  
              }else{
                  break;
              }
              
              c++;
          }
          
          c = 1;
          while(true){
              
              if(c > PAGE_COUNT){
                  break;
              }
              
              doc = Jsoup.connect(ZZYJ_PROJECT_SUPPORTER+proCode+"/expert/"+c).timeout(CON_TIMEOUT).get();
              els = doc.select("div.commbox");
              _type = 0;
              if(els.select("div.fl").size() > 0){
                  
                  for(Element _el :els){
                      
                      els = _el.select("img.avater");
                      if(els.size() > 0){
                          _image = els.first().attr("src");
                          if(_image.contains("http")){
                              _image = ZZYJ_SITE + _image.trim(); 
                          }
                      }
                      
                      els = _el.select("div").last().select("p");
                      if(els.size() > 0){
                          _time = getTimeFromStrByF2(els.first().text(),"yyyy-MM-dd hh:mm:ss");
                      }
                      
                      els = _el.select("a[href *= /user/qzone]");
                      if(els.size() > 0){
                          _expertCode = this.getCodeFromUrl(els.first().attr("href"));
                          _nickName = els.first().text();
                          
                          if(els.first().select("span.vtd2 img").size() > 0){
                              _authIcon = ZZYJ_SITE+els.first().select("span.vtd2 img").first().attr("src");
                          }
                          
                          els = els.first().parent().parent().select("p").last().select("span");
                          if(els.size() > 0){
                              if(StringUtil.isNotBlank(els.first().text().trim())){
                                  _score = Integer.valueOf(els.first().text().trim());  
                              }
                          }
                      }
                      
                      voList.add(new ProjectSupporter(_projectCode, _expertCode, _time, _score, _type,_image.trim(),_authIcon.trim(),_nickName.trim()));
                      //System.out.println(_nickName+";"+_authIcon+";"+_image+";"+_projectCode+";"+_expertCode+";"+_time+";"+_score+";"+_type);
                  }
                  
              }else{
                  break;
              }
              
              c++;
          }
             
         } catch (IOException e) {
             e.printStackTrace();
             return 0;
         }
         
         return 1;
         
    }
    
    private String cutStrByReg(String text, String reg){
        
        if(StringUtil.isNotBlank(text)){
            if(text.contains(reg)){
                return text.substring(text.indexOf(reg));
            }
        }
        
        return "";
    }
    
    private String getCodeFromUrl(String url){
        if(StringUtil.isNotBlank(url)){
            return url.substring(url.lastIndexOf("/")+1);
        }
        return "";
    }
    
    private String getPostStringOfColon(String str){
        if(StringUtil.isNotBlank(str)){
            str = str.replace(":", "#").replace("：","#");
            String temp = str.substring(str.indexOf("#")+1);
            return temp;
        }
        return null;
    }
    
    private Date getTimeFromStrByF(String dateStr,String formate){
        if(StringUtil.isNotBlank(dateStr)){
            dateStr = dateStr.replace(":", "#").replace("：","#");
            String temp = dateStr.substring(dateStr.indexOf("#")+1);
            SimpleDateFormat sf = new SimpleDateFormat(formate);
            try {
                return sf.parse(temp);
            } catch (ParseException e) {
                e.printStackTrace();
            }
        }
        
        return null;
    }
    
    private Date getTimeFromStrByF2(String dateStr,String formate){
        if(StringUtil.isNotBlank(dateStr)){
            
            SimpleDateFormat sf = new SimpleDateFormat(formate);
            try {
                return sf.parse(dateStr.trim());
            } catch (ParseException e) {
                e.printStackTrace();
            }
        }
        
        return null;
    }
    
    public void updateSpideData(){
        spideData();
        zzyjSpiderService.updateSpideData(voList);
    }

}
